from NewsMining.db.dbConnect import getDBConnection
from mynewsweb.model import *
from math import sqrt
from NewsMining.crawler.preprocessText import *
from NewsMining.db import dbConnect
from cache import cached_kernel

#==== ==== ==== ====
# Settings
#==== ==== ==== ====
k = 8

#==== ==== ==== ====
# CLEAR RESULTS
#==== ==== ==== ====
db = getDBConnection()

##previus_results = db.query(KernelResults.news_id_1, KernelResults.news_id_2).\
##                    filter(and_(KernelResults.kernel_id == 1, KernelResults.param1 == k)).all()
##
##for r in previus_results:
##    pass #db.delete(r)
##
##db.commit()
#sdfsdfsdfsdf
#==== ==== ==== ====
# NGRAM KERNEL
#==== ==== ==== ====
def ngram_kernel(s1, s2, data, k=0, decay=0):
    '''simple n-gram string kernel'''
    s1d = data[s1]
    s2d = data[s2]
    inter = set(s1d.keys()).intersection(s2d.keys())

    return sum(s1d[i] * s2d[i] for i in inter)

#==== ==== ==== ====
# DATA
#==== ==== ==== ====
print "Prepare data!"
news = db.query(News).filter(News.rank != None).all()
news_data = {}

for n in news:
    text = removeStopWords(preprocess(n.content))
    if len(text) > 300:
	    kmers = [text[i:i+k] for i in range(len(text)-k+1)]
	    d = dict([(i, 0) for i in kmers])

	    for i in kmers:
             d[i] += 1

	    news_data[n.news_id] = d #[text[i:i+k] for i in range(len(text)-k+1)]

#==== ==== ==== ====
# CALL
#==== ==== ==== ====
print "Run kernel"
kernel = cached_kernel(ngram_kernel)

# One vs ALL
##n1 = 66 #249, 27, 194, 66
##for n2 in news_data:
##    if n1 != n2:
##        print n1, n2
##        d = kernel(n1, n2, news_data) / sqrt(kernel(n1, n1, news_data) * kernel(n2, n2, news_data))
##        if d > 0:
##            r = KernelResults(n1, n2, d, 1)
##            db.add(r)

# ALL vs ALL
i = 0
for n1 in news_data.keys()[:]:
    print n1
    for n2 in news_data:
        if n1 != n2:
            d = kernel(n1, n2, news_data) / sqrt(kernel(n1, n1, news_data) * kernel(n2, n2, news_data))
            if d > 0:
                r = KernelResults(n1, n2, d, 1, k, 0)
                db.add(r)

    i += 1
    if i % 100 == 0:
        print "Commit"
        db.commit()

db.commit()